from typing import Set

import requests
from pyquery import PyQuery as pq
import xlwt

url = "https://space.bilibili.com/928123/video?tid=0&page=1&keyword=&order=pubdate"

urls: str = {
    "https://space.bilibili.com/928123/video?tid=0&page=1&keyword=&order=pubdate"
}

headers = {
    'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.6241 SLBChan/27'
}

response = requests.get(url=url, headers=headers)

for i in range(2, 10):
    url = url + "&page=" + str(i)
    urls = []
    urls.append(url)

    url = "https://space.bilibili.com/928123/video?tid=0&page=1&keyword=&order=pubdate"

data = []
datalist = []


for url in urls:
    response = requests.get(url=url, headers=headers).text
    doc = pq(response)
    lis = doc('.body-contain li').items()

    for li in lis:
        #提取标题
        title = li('.title').text()
        data.append(title)
        #播放量
        plays = li('.meta span.play').text()
        data.append(plays)
        #上传时间
        data = li('.meta span.time').text()
        data.append(data)

        datalist.append(data)

        data = []

print(datalist)

book = xlwt.Workbook(encoding="utf-8", style_compression=0)
sheet = book.add_sheet('番剧(8', cell_overwrite_ok=True)

col = ("番剧名", "播放量", "上映日期")  #表头

for i in range(0, 3):
    sheet.write(0, i, col[i])

for i in range(0, 300):
    try:
        a = 'datalist[i]'
    except:
        continue
for j in range(0, 4):
    sheet.write(i+1, j, a[j])
#-------
import mitmproxy
# 待执行的 JS 代码,修改 window.navigator.webdriver 的值
js_exec = 'Object.defineProperties(navigator,{webdriver:{get:() => false}});'


# 重写 response，截获网络请求，js注入
def response(slef, flow: mitmproxy.http.HTTPFlow):
    if 'google' in flow.request.url:
        flow.response.text = js_exec + flow.response.text
        # 启动mitmproxy
        mitmdump - p_8888 - s_111.py
        # 配置ChromeOptions
        option.add_argument("--proxy-server=http://127.0.0.1:8888")

book.save("番剧数据(8.xls")